
# Sources
# https://matheusfacure.github.io/python-causality-handbook/15-Synthetic-Control.html
# https://nbviewer.jupyter.org/github/OscarEngelbrektson/SyntheticControlMethods/blob/master/examples/user_guide.ipynb


import warnings
import imp
import ascm # get module reference for reload
import copy
import numpy as np
import os 
import pandas as pd
import seaborn as sns
import statsmodels.formula.api as smf
from ascm.graph import graph_original
from ascm.graph import ascm_dataframe
from ascm.graph import ascm_visualize
from ascm.graph import rmspe_table
from ascm.graph import p_value
from ascm.graph import placebo
from ascm.graph import ascm_limit_donor_pool
from matplotlib import style
from matplotlib import pyplot as plt
from matplotlib.colors import LogNorm
from operator import add
from scipy import stats
from scipy.optimize import fmin_slsqp
from scipy.stats import ttest_ind
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression,Ridge
from toolz import reduce, partial
from typing import List


imp.reload(ascm.graph)
warnings.filterwarnings('ignore')
%matplotlib inline
pd.set_option("display.max_columns", 6)
style.use("fivethirtyeight")

# read data file
# pandas read_csv
dirs = os.getcwd() 
files = os.listdir(dirs) 

data = pd.read_excel(r'/Users/jinhyun/Documents/GitHub/Python/UvA projects/Thesis/MSc/data/processed raw data_2.xlsx') 
data = data.loc[ (data.year < 2021) & (data.year >= 1965)]
raw_data = copy.deepcopy(data)# Copy dataframe


data_gdp = data[
['year','country','GDP (current US$)',
 'Foreign direct investment, net inflows (BoP, current US$)','GDP growth (annual %)','Life expectancy at birth, total (years)'
 ,'Agriculture, forestry, and fishing, value added (% of GDP)'
 ,'Adolescent fertility rate (births per 1,000 women ages 15-19)'
 ,'Inflation, GDP deflator (annual %)'
 ,'School enrollment, primary (% gross)'
 ,'Fertility rate, total (births per woman)'
 , 'Surface area (sq. km)'
 ,'Mortality rate, under-5 (per 1,000 live births)']]

data_gdp_growth = data[
    ['year','country','GDP growth (annual %)','Industry (including construction), value added (% of GDP)',
    'Life expectancy at birth, total (years)','Inflation, GDP deflator (annual %)']]


data_inv = data[
    ['year','country','Gross capital formation (% of GDP)',
       'Industry (including construction), value added (% of GDP)',
       'Adolescent fertility rate (births per 1,000 women ages 15-19)',
       'Inflation, GDP deflator (annual %)',
       'Merchandise trade (% of GDP)',
       'Agriculture, forestry, and fishing, value added (% of GDP)',
       'School enrollment, primary (% gross)']]

data_export = data[
    ['year','country', 'Exports of goods and services (% of GDP)',
       'Industry (including construction), value added (% of GDP)',
       'Adolescent fertility rate (births per 1,000 women ages 15-19)',
       'Inflation, GDP deflator (annual %)',
       'Surface area (sq. km)',
       'Merchandise trade (% of GDP)',
       'Life expectancy at birth, total (years)',
       'School enrollment, primary (% gross)']]


data_import = data[
    ['year','country', 'Imports of goods and services (% of GDP)',
       'Industry (including construction), value added (% of GDP)',
       'Adolescent fertility rate (births per 1,000 women ages 15-19)',
       'Inflation, GDP deflator (annual %)',
       'Surface area (sq. km)',
       'Merchandise trade (% of GDP)',
       'Life expectancy at birth, total (years)',
       'School enrollment, primary (% gross)']]
       
data_fdi = data[
    ['year','country', 'Foreign direct investment, net inflows (BoP, current US$)',
       'Adolescent fertility rate (births per 1,000 women ages 15-19)',
       'Agriculture, forestry, and fishing, value added (% of GDP)',
       'Exports of goods and services (% of GDP)',
       'Fertility rate, total (births per woman)',
       'GDP (current US$)', 'GDP growth (annual %)',
       'Imports of goods and services (% of GDP)',
       'Industry (including construction), value added (% of GDP)',
       'Inflation, GDP deflator (annual %)',
       'Life expectancy at birth, total (years)',
       'Merchandise trade (% of GDP)',
       'Mortality rate, under-5 (per 1,000 live births)',
       'Population density (people per sq. km of land area)',
        'Population, total',
       'School enrollment, primary (% gross)', 'Surface area (sq. km)',
       'Urban population growth (annual %)' ]]


display(data_gdp)


display(data_gdp_growth)


display(data_inv)


display(data_export)


display(data_import)


display(data_fdi)


graph_original('European Union','GDP (current US$)', data_gdp)


graph_original('European Union','GDP growth (annual %)',data_gdp_growth)


graph_original('European Union','Exports of goods and services (% of GDP)', data_export)


graph_original('European Union','Imports of goods and services (% of GDP)', data_import)


graph_original('European Union','Foreign direct investment, net inflows (BoP, current US$)', data_fdi)


graph_original('European Union','Gross capital formation (% of GDP)', data_inv)


ascm_dataframe('European Union','GDP (current US$)', data_gdp, incident= 'QE', incident_year=2009)

Pre- RMSPE : 0.672182387459166
Post- RMSPE : 1.7296717596378866


ascm_dataframe('European Union','GDP growth (annual %)', data_gdp_growth, incident= 'QE', incident_year=2009)

Pre- RMSPE : 1.9642319552773726
Post- RMSPE : 23.30646654554046


ascm_dataframe('European Union','Exports of goods and services (% of GDP)', data_export, incident= 'QE', incident_year=2009)

Pre- RMSPE : 0.1418782476654889
Post- RMSPE : 0.21279190540557036


ascm_dataframe('European Union','Imports of goods and services (% of GDP)', data_import, incident= 'QE', incident_year=2009)

Pre- RMSPE : 0.09023993688066338
Post- RMSPE : 0.13278616557844877


ascm_dataframe('European Union','Gross capital formation (% of GDP)', data_inv, incident= 'QE', incident_year=2009)

Pre- RMSPE : 0.10039886536924084
Post- RMSPE : 0.10750962227906978


ascm_dataframe('European Union', 'Foreign direct investment, net inflows (BoP, current US$)', data_fdi, incident= 'QE', incident_year=2009)

Pre- RMSPE : 0.7663515731978118
Post- RMSPE : 2.385431102871565


ascm_visualize('European Union','GDP (current US$)', data_gdp, incident= 'QE', incident_year=2009, std_scaling=True, ylabel=False)


ascm_visualize('European Union','GDP growth (annual %)', data_gdp_growth, incident= 'QE', incident_year=2009, std_scaling=True, ylabel=False)


ascm_visualize('European Union','Exports of goods and services (% of GDP)', data_export, incident= 'QE', incident_year=2009, std_scaling=True, ylabel=False)


ascm_visualize('European Union','Imports of goods and services (% of GDP)', data_import, incident= 'QE', incident_year=2009, std_scaling=True, ylabel=False)


ascm_visualize('European Union','Gross capital formation (% of GDP)', data_inv, incident= 'QE', incident_year=2009, std_scaling=True, ylabel=False)


ascm_visualize('European Union','Foreign direct investment, net inflows (BoP, current US$)', data_fdi, incident= 'QE', incident_year=2009, std_scaling=False, ylabel="Foreign direct investment, US$ trillion")


rmspe_table('European Union', 'GDP (current US$)', data_gdp,  incident= 'QE' ,incident_year=2009, outlier = 1)


rmspe_table('European Union', 'GDP growth (annual %)', data_gdp_growth,  incident= 'QE' ,incident_year=2009, outlier = 1)


rmspe_table('European Union', 'Exports of goods and services (% of GDP)', data_export,  incident= 'QE' ,incident_year=2009, outlier = 2)


rmspe_table('European Union', 'Imports of goods and services (% of GDP)', data_import,  incident= 'QE' ,incident_year=2009, outlier = 2)


rmspe_table('European Union', 'Gross capital formation (% of GDP)', data_inv,  incident= 'QE' ,incident_year=2009, outlier = 0)


rmspe_table('European Union','Foreign direct investment, net inflows (BoP, current US$)', data_fdi,  incident= 'QE' ,incident_year=2009, outlier = 0)


placebo('European Union','GDP (current US$)', data_gdp, incident_year = 2009, ylabel="GDP US$ 10 trillion" )


placebo('European Union','GDP growth (annual %)', data_gdp_growth, incident_year = 2009, ylabel="GDP US$ 10 trillion" )


placebo('European Union','Exports of goods and services (% of GDP)', data_export, incident_year = 2009, ylabel="GDP US$ 10 trillion" )


placebo('European Union', 'Imports of goods and services (% of GDP)', data_import, incident_year = 2009, ylabel="GDP US$ 10 trillion" )


placebo('European Union','Gross capital formation (% of GDP)', data_inv, incident_year = 2009, ylabel="GDP US$ 10 trillion" )


placebo('European Union','Foreign direct investment, net inflows (BoP, current US$)', data_fdi, incident_year = 2009, ylabel="GDP US$ 10 trillion" )


p_value('European Union','GDP (current US$)', data_gdp, incident_year = 2009, detail= False)


p_value('European Union','GDP growth (annual %)', data_gdp_growth, incident_year = 2009, detail= False)


p_value('European Union','Exports of goods and services (% of GDP)', data_export, incident_year = 2009, detail= False)


p_value('European Union','Imports of goods and services (% of GDP)', data_import, incident_year = 2009, detail= False)


p_value('European Union', 'Gross capital formation (% of GDP)', data_inv,  incident_year = 2009, detail= False)


p_value('European Union','Foreign direct investment, net inflows (BoP, current US$)', data_fdi,  incident_year = 2009, detail= False)


ascm_limit_donor_pool('European Union','GDP (current US$)', data_gdp, incident_year = 2009, std_scaling=True, detail = False, ylabel="GDP US$ 10 trillion")


ascm_limit_donor_pool('European Union','GDP growth (annual %)', data_gdp_growth, incident_year = 2009, std_scaling=True, detail = False, ylabel=False)


ascm_limit_donor_pool('European Union', 'Exports of goods and services (% of GDP)', data_export,  incident_year = 2009, std_scaling=True, detail = False, ylabel=False)


ascm_limit_donor_pool('European Union','Imports of goods and services (% of GDP)', data_import,  incident_year = 2009, std_scaling=True, detail = False, ylabel=False)


ascm_limit_donor_pool('European Union', 'Gross capital formation (% of GDP)', data_inv, incident_year = 2009, std_scaling=True, detail = False, ylabel=False)


ascm_limit_donor_pool('European Union','Foreign direct investment, net inflows (BoP, current US$)', data_fdi, incident_year = 2009, std_scaling=True, detail = False, ylabel=False)

	year	country	GDP (current US$)	...	Fertility rate, total (births per woman)	Surface area (sq. km)	Mortality rate, under-5 (per 1,000 live births)
5	1965	Algeria	3.136259e+09	...	7.675000	2381740.000	244.500000
6	1966	Algeria	3.039835e+09	...	7.676000	2381740.000	245.200000
7	1967	Algeria	3.370843e+09	...	7.672000	2381740.000	245.200000
8	1968	Algeria	3.852116e+09	...	7.666000	2381740.000	244.200000
9	1969	Algeria	4.257219e+09	...	7.656000	2381740.000	242.000000
...	...	...	...	...	...	...	...
4570	2016	European Union	1.389326e+13	...	1.566241	4172271.600	4.165583
4571	2017	European Union	1.476599e+13	...	1.556049	4172076.830	4.110764
4572	2018	European Union	1.597821e+13	...	1.542059	4172838.889	4.031467
4573	2019	European Union	1.568936e+13	...	1.525078	18270.000	3.943376
4574	2020	European Union	1.529193e+13	...	1.495717	18270.000	3.909295

	year	country	GDP growth (annual %)	Industry (including construction), value added (% of GDP)	Life expectancy at birth, total (years)	Inflation, GDP deflator (annual %)
5	1965	Algeria	6.206898	35.860933	48.394000	1.501331
6	1966	Algeria	-4.804971	35.860933	48.811000	1.817815
7	1967	Algeria	9.452963	35.860933	49.210000	1.312041
8	1968	Algeria	10.796239	35.860933	49.597000	3.142056
9	1969	Algeria	8.433280	35.860933	49.982000	1.921085
...	...	...	...	...	...	...
4570	2016	European Union	2.008266	22.793653	80.949865	0.874115
4571	2017	European Union	2.813595	22.723281	80.930839	1.464064
4572	2018	European Union	2.065499	22.704654	81.027024	2.002209
4573	2019	European Union	1.816850	22.646120	81.313270	2.199604
4574	2020	European Union	-5.955778	22.513421	80.463702	1.596823

	year	country	Gross capital formation (% of GDP)	...	Merchandise trade (% of GDP)	Agriculture, forestry, and fishing, value added (% of GDP)	School enrollment, primary (% gross)
5	1965	Algeria	22.603944	...	41.705741	11.106980	74.465683
6	1966	Algeria	17.324132	...	41.581210	11.106980	74.465683
7	1967	Algeria	23.434417	...	40.405322	11.106980	74.465683
8	1968	Algeria	27.867922	...	42.703804	11.106980	74.465683
9	1969	Algeria	32.352736	...	45.640126	11.106980	74.465683
...	...	...	...	...	...	...	...
4570	2016	European Union	20.947080	...	69.037866	1.587936	101.602577
4571	2017	European Union	21.569979	...	71.975157	1.693777	101.832970
4572	2018	European Union	22.146677	...	73.506501	1.609880	101.509720
4573	2019	European Union	22.934326	...	72.454613	1.591050	101.294121
4574	2020	European Union	22.069685	...	69.324204	1.651606	106.739441

	year	country	Exports of goods and services (% of GDP)	...	Merchandise trade (% of GDP)	Life expectancy at birth, total (years)	School enrollment, primary (% gross)
5	1965	Algeria	22.603944	...	41.705741	48.394000	74.465683
6	1966	Algeria	25.986198	...	41.581210	48.811000	74.465683
7	1967	Algeria	23.434417	...	40.405322	49.210000	74.465683
8	1968	Algeria	23.135635	...	42.703804	49.597000	74.465683
9	1969	Algeria	23.788777	...	45.640126	49.982000	74.465683
...	...	...	...	...	...	...	...
4570	2016	European Union	46.684512	...	69.037866	80.949865	101.602577
4571	2017	European Union	48.354594	...	71.975157	80.930839	101.832970
4572	2018	European Union	49.198138	...	73.506501	81.027024	101.509720
4573	2019	European Union	49.344668	...	72.454613	81.313270	101.294121
4574	2020	European Union	46.648574	...	69.324204	80.463702	106.739441

	year	country	Imports of goods and services (% of GDP)	...	Merchandise trade (% of GDP)	Life expectancy at birth, total (years)	School enrollment, primary (% gross)
5	1965	Algeria	25.833078	...	41.705741	48.394000	74.465683
6	1966	Algeria	24.653572	...	41.581210	48.811000	74.465683
7	1967	Algeria	21.631769	...	40.405322	49.210000	74.465683
8	1968	Algeria	24.187254	...	42.703804	49.597000	74.465683
9	1969	Algeria	28.070758	...	45.640126	49.982000	74.465683
...	...	...	...	...	...	...	...
4570	2016	European Union	42.306978	...	69.037866	80.949865	101.602577
4571	2017	European Union	44.077874	...	71.975157	80.930839	101.832970
4572	2018	European Union	45.325864	...	73.506501	81.027024	101.509720
4573	2019	European Union	45.897118	...	72.454613	81.313270	101.294121
4574	2020	European Union	42.871214	...	69.324204	80.463702	106.739441

1. Data processing¶

Result¶

1. Raw data¶

1.2 Dataframe of Difference¶

1.3 Augmented Synthetic Control Method¶

1.4 RMSPE ratio¶

1.5 placebo effect analysis¶

1.6 p-value analysis¶

1.7 Limitation of donor pool test¶

	year	country	Foreign direct investment, net inflows (BoP, current US$)	...	School enrollment, primary (% gross)	Surface area (sq. km)	Urban population growth (annual %)
5	1965	Algeria	8.012000e+07	...	74.465683	2381740.000	6.729878
6	1966	Algeria	8.012000e+07	...	74.465683	2381740.000	5.894366
7	1967	Algeria	8.012000e+07	...	74.465683	2381740.000	3.266680
8	1968	Algeria	8.012000e+07	...	74.465683	2381740.000	3.307126
9	1969	Algeria	8.012000e+07	...	74.465683	2381740.000	3.292629
...	...	...	...	...	...	...	...
4570	2016	European Union	7.297543e+11	...	101.602577	4172271.600	0.479309
4571	2017	European Union	5.400017e+11	...	101.832970	4172076.830	0.430422
4572	2018	European Union	-3.415352e+10	...	101.509720	4172838.889	0.445310
4573	2019	European Union	1.783994e+11	...	101.294121	18270.000	0.359546
4574	2020	European Union	2.960592e+11	...	106.739441	18270.000	0.359835

	country	year	GDP (current US$)	ASCM	Difference
4519	European Union	1965	7.259311e+11	1.976478e+11	5.282834e+11
4520	European Union	1966	7.259311e+11	2.304571e+11	4.954740e+11
4521	European Union	1967	7.259311e+11	2.078856e+11	5.180455e+11
4522	European Union	1968	7.259311e+11	1.954812e+11	5.304499e+11
4523	European Union	1969	7.259311e+11	2.175651e+11	5.083660e+11
4524	European Union	1970	7.259311e+11	2.537035e+11	4.722276e+11
4525	European Union	1971	8.208130e+11	2.772500e+11	5.435630e+11
4526	European Union	1972	9.904287e+11	3.311914e+11	6.592373e+11
4527	European Union	1973	1.282021e+12	4.246519e+11	8.573687e+11
4528	European Union	1974	1.451750e+12	3.850346e+11	1.066715e+12
4529	European Union	1975	1.690875e+12	5.041165e+11	1.186759e+12
4530	European Union	1976	1.770054e+12	4.283603e+11	1.341694e+12
4531	European Union	1977	2.004900e+12	4.640460e+11	1.540855e+12
4532	European Union	1978	2.442696e+12	3.751789e+11	2.067518e+12
4533	European Union	1979	2.952838e+12	5.026880e+11	2.450150e+12
4534	European Union	1980	3.303182e+12	6.766747e+11	2.626507e+12
4535	European Union	1981	2.879984e+12	7.009525e+11	2.179032e+12
4536	European Union	1982	2.777377e+12	4.355578e+11	2.341819e+12
4537	European Union	1983	2.702070e+12	5.258286e+11	2.176241e+12
4538	European Union	1984	2.603395e+12	6.434572e+11	1.959938e+12
4539	European Union	1985	2.677661e+12	9.403014e+11	1.737360e+12
4540	European Union	1986	3.744042e+12	7.622508e+11	2.981791e+12
4541	European Union	1987	4.631251e+12	6.459640e+11	3.985287e+12
4542	European Union	1988	5.084288e+12	6.505909e+11	4.433697e+12
4543	European Union	1989	5.193669e+12	6.168973e+11	4.576772e+12
4544	European Union	1990	6.498472e+12	7.218652e+11	5.776607e+12
4545	European Union	1991	6.736016e+12	8.829104e+11	5.853106e+12
4546	European Union	1992	7.406290e+12	1.170741e+12	6.235549e+12
4547	European Union	1993	6.761115e+12	1.616510e+12	5.144605e+12
4548	European Union	1994	7.161872e+12	2.055437e+12	5.106435e+12
4549	European Union	1995	8.296026e+12	2.042153e+12	6.253873e+12
4550	European Union	1996	8.431518e+12	2.560011e+12	5.871507e+12
4551	European Union	1997	7.733761e+12	3.094841e+12	4.638921e+12
4552	European Union	1998	7.969786e+12	3.776813e+12	4.192973e+12
4553	European Union	1999	7.925751e+12	4.168291e+12	3.757460e+12
4554	European Union	2000	7.276100e+12	4.714951e+12	2.561149e+12
4555	European Union	2001	7.393823e+12	5.518455e+12	1.875368e+12
4556	European Union	2002	8.083838e+12	6.085657e+12	1.998181e+12
4557	European Union	2003	9.931880e+12	6.407244e+12	3.524636e+12
4558	European Union	2004	1.141903e+13	7.101929e+12	4.317098e+12
4559	European Union	2005	1.191085e+13	8.300035e+12	3.610811e+12
4560	European Union	2006	1.271392e+13	1.010002e+13	2.613904e+12
4561	European Union	2007	1.472876e+13	1.303778e+13	1.690981e+12
4562	European Union	2008	1.629763e+13	1.676312e+13	-4.654938e+11
4563	European Union	2009	1.476378e+13	1.845278e+13	-3.688997e+12
4564	European Union	2010	1.455764e+13	2.166847e+13	-7.110831e+12
4565	European Union	2011	1.575949e+13	2.662600e+13	-1.086650e+13
4566	European Union	2012	1.463620e+13	2.989827e+13	-1.526207e+13
4567	European Union	2013	1.529973e+13	3.398700e+13	-1.868727e+13
4568	European Union	2014	1.565210e+13	3.777348e+13	-2.212139e+13
4569	European Union	2015	1.355193e+13	3.964489e+13	-2.609297e+13
4570	European Union	2016	1.389326e+13	4.052396e+13	-2.663070e+13
4571	European Union	2017	1.476599e+13	4.431011e+13	-2.954412e+13
4572	European Union	2018	1.597821e+13	5.029363e+13	-3.431542e+13
4573	European Union	2019	1.568936e+13	5.205010e+13	-3.636074e+13
4574	European Union	2020	1.529193e+13	5.352401e+13	-3.823208e+13